{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import os\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pickle as pkl\n",
    "from tqdm import tqdm\n",
    "\n",
    "from moment.utils.experiment_utils import \\\n",
    "    get_dl4tsc_results, get_ts2vec_results, draw_cd_diagram"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Results from TS2Vec and DL4TSC on UCR datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TS2Vec</th>\n",
       "      <th>T-Loss</th>\n",
       "      <th>TNC</th>\n",
       "      <th>TS-TCC</th>\n",
       "      <th>TST</th>\n",
       "      <th>DTW</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Adiac</th>\n",
       "      <td>0.762</td>\n",
       "      <td>0.675</td>\n",
       "      <td>0.726</td>\n",
       "      <td>0.767</td>\n",
       "      <td>0.550</td>\n",
       "      <td>0.604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArrowHead</th>\n",
       "      <td>0.857</td>\n",
       "      <td>0.766</td>\n",
       "      <td>0.703</td>\n",
       "      <td>0.737</td>\n",
       "      <td>0.771</td>\n",
       "      <td>0.703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Beef</th>\n",
       "      <td>0.767</td>\n",
       "      <td>0.667</td>\n",
       "      <td>0.733</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.500</td>\n",
       "      <td>0.633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BeetleFly</th>\n",
       "      <td>0.900</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.8</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BirdChicken</th>\n",
       "      <td>0.800</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.650</td>\n",
       "      <td>0.750</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             TS2Vec T-Loss    TNC TS-TCC    TST    DTW\n",
       "Dataset                                               \n",
       "Adiac         0.762  0.675  0.726  0.767  0.550  0.604\n",
       "ArrowHead     0.857  0.766  0.703  0.737  0.771  0.703\n",
       "Beef          0.767  0.667  0.733    0.6  0.500  0.633\n",
       "BeetleFly     0.900    0.8   0.85    0.8  1.000  0.700\n",
       "BirdChicken   0.800   0.85   0.75   0.65  0.650  0.750"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ucr_results_unsupervised = get_ts2vec_results(database=\"ucr\")\n",
    "ucr_results_unsupervised.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TS2Vec</th>\n",
       "      <th>T-Loss</th>\n",
       "      <th>TNC</th>\n",
       "      <th>TS-TCC</th>\n",
       "      <th>TST</th>\n",
       "      <th>DTW</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ArticularyWordRecognition</th>\n",
       "      <td>0.987</td>\n",
       "      <td>0.943</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.953</td>\n",
       "      <td>0.977</td>\n",
       "      <td>0.987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AtrialFibrillation</th>\n",
       "      <td>0.200</td>\n",
       "      <td>0.133</td>\n",
       "      <td>0.133</td>\n",
       "      <td>0.267</td>\n",
       "      <td>0.067</td>\n",
       "      <td>0.200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BasicMotions</th>\n",
       "      <td>0.975</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.975</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.975</td>\n",
       "      <td>0.975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CharacterTrajectories</th>\n",
       "      <td>0.995</td>\n",
       "      <td>0.993</td>\n",
       "      <td>0.967</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.975</td>\n",
       "      <td>0.989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cricket</th>\n",
       "      <td>0.972</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.958</td>\n",
       "      <td>0.917</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           TS2Vec  T-Loss    TNC  TS-TCC    TST    DTW\n",
       "Dataset                                                               \n",
       "ArticularyWordRecognition   0.987   0.943  0.973   0.953  0.977  0.987\n",
       "AtrialFibrillation          0.200   0.133  0.133   0.267  0.067  0.200\n",
       "BasicMotions                0.975   1.000  0.975   1.000  0.975  0.975\n",
       "CharacterTrajectories       0.995   0.993  0.967   0.985  0.975  0.989\n",
       "Cricket                     0.972   0.972  0.958   0.917  1.000  1.000"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uea_results_unsupervised = get_ts2vec_results(database=\"uea\")\n",
    "uea_results_unsupervised.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CNN</th>\n",
       "      <th>Encoder</th>\n",
       "      <th>FCN</th>\n",
       "      <th>MCDNN</th>\n",
       "      <th>MLP</th>\n",
       "      <th>ResNet</th>\n",
       "      <th>t-LeNet</th>\n",
       "      <th>TWIESN</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ACSF1</th>\n",
       "      <td>0.334000</td>\n",
       "      <td>0.444000</td>\n",
       "      <td>0.898000</td>\n",
       "      <td>0.226000</td>\n",
       "      <td>0.558000</td>\n",
       "      <td>0.916000</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.592000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Adiac</th>\n",
       "      <td>0.393350</td>\n",
       "      <td>0.318159</td>\n",
       "      <td>0.841432</td>\n",
       "      <td>0.620460</td>\n",
       "      <td>0.391304</td>\n",
       "      <td>0.833248</td>\n",
       "      <td>0.022506</td>\n",
       "      <td>0.427621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AllGestureWiimoteX</th>\n",
       "      <td>0.411143</td>\n",
       "      <td>0.475143</td>\n",
       "      <td>0.713429</td>\n",
       "      <td>0.261429</td>\n",
       "      <td>0.476571</td>\n",
       "      <td>0.740571</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.522000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AllGestureWiimoteY</th>\n",
       "      <td>0.478857</td>\n",
       "      <td>0.509429</td>\n",
       "      <td>0.784286</td>\n",
       "      <td>0.419714</td>\n",
       "      <td>0.570571</td>\n",
       "      <td>0.793714</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.600286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AllGestureWiimoteZ</th>\n",
       "      <td>0.375143</td>\n",
       "      <td>0.396000</td>\n",
       "      <td>0.692000</td>\n",
       "      <td>0.287143</td>\n",
       "      <td>0.439143</td>\n",
       "      <td>0.725714</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.516286</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         CNN   Encoder       FCN     MCDNN       MLP  \\\n",
       "Dataset                                                                \n",
       "ACSF1               0.334000  0.444000  0.898000  0.226000  0.558000   \n",
       "Adiac               0.393350  0.318159  0.841432  0.620460  0.391304   \n",
       "AllGestureWiimoteX  0.411143  0.475143  0.713429  0.261429  0.476571   \n",
       "AllGestureWiimoteY  0.478857  0.509429  0.784286  0.419714  0.570571   \n",
       "AllGestureWiimoteZ  0.375143  0.396000  0.692000  0.287143  0.439143   \n",
       "\n",
       "                      ResNet   t-LeNet    TWIESN  \n",
       "Dataset                                           \n",
       "ACSF1               0.916000  0.100000  0.592000  \n",
       "Adiac               0.833248  0.022506  0.427621  \n",
       "AllGestureWiimoteX  0.740571  0.100000  0.522000  \n",
       "AllGestureWiimoteY  0.793714  0.100000  0.600286  \n",
       "AllGestureWiimoteZ  0.725714  0.100000  0.516286  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ucr_results_supervised = get_dl4tsc_results(database=\"ucr\")\n",
    "ucr_results_supervised.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CNN</th>\n",
       "      <th>Encoder</th>\n",
       "      <th>FCN</th>\n",
       "      <th>MCDNN</th>\n",
       "      <th>MCNN</th>\n",
       "      <th>MLP</th>\n",
       "      <th>ResNet</th>\n",
       "      <th>t-LeNet</th>\n",
       "      <th>TWIESN</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>50words</th>\n",
       "      <td>0.620879</td>\n",
       "      <td>0.723297</td>\n",
       "      <td>0.627473</td>\n",
       "      <td>0.589451</td>\n",
       "      <td>0.219560</td>\n",
       "      <td>0.684396</td>\n",
       "      <td>0.739560</td>\n",
       "      <td>0.125275</td>\n",
       "      <td>0.496044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Adiac</th>\n",
       "      <td>0.379028</td>\n",
       "      <td>0.484143</td>\n",
       "      <td>0.843990</td>\n",
       "      <td>0.610486</td>\n",
       "      <td>0.021995</td>\n",
       "      <td>0.396675</td>\n",
       "      <td>0.828900</td>\n",
       "      <td>0.020460</td>\n",
       "      <td>0.416368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArrowHead</th>\n",
       "      <td>0.722857</td>\n",
       "      <td>0.804000</td>\n",
       "      <td>0.842857</td>\n",
       "      <td>0.684571</td>\n",
       "      <td>0.339429</td>\n",
       "      <td>0.778286</td>\n",
       "      <td>0.844571</td>\n",
       "      <td>0.302857</td>\n",
       "      <td>0.658857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Beef</th>\n",
       "      <td>0.763333</td>\n",
       "      <td>0.643333</td>\n",
       "      <td>0.696667</td>\n",
       "      <td>0.563333</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.720000</td>\n",
       "      <td>0.753333</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.536667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BeetleFly</th>\n",
       "      <td>0.890000</td>\n",
       "      <td>0.745000</td>\n",
       "      <td>0.860000</td>\n",
       "      <td>0.580000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.870000</td>\n",
       "      <td>0.850000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.730000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                CNN   Encoder       FCN     MCDNN      MCNN       MLP  \\\n",
       "Dataset                                                                 \n",
       "50words    0.620879  0.723297  0.627473  0.589451  0.219560  0.684396   \n",
       "Adiac      0.379028  0.484143  0.843990  0.610486  0.021995  0.396675   \n",
       "ArrowHead  0.722857  0.804000  0.842857  0.684571  0.339429  0.778286   \n",
       "Beef       0.763333  0.643333  0.696667  0.563333  0.200000  0.720000   \n",
       "BeetleFly  0.890000  0.745000  0.860000  0.580000  0.500000  0.870000   \n",
       "\n",
       "             ResNet   t-LeNet    TWIESN  \n",
       "Dataset                                  \n",
       "50words    0.739560  0.125275  0.496044  \n",
       "Adiac      0.828900  0.020460  0.416368  \n",
       "ArrowHead  0.844571  0.302857  0.658857  \n",
       "Beef       0.753333  0.200000  0.536667  \n",
       "BeetleFly  0.850000  0.500000  0.730000  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uea_results_supervised = get_dl4tsc_results(database=\"uea\")\n",
    "uea_results_supervised.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TS2Vec</th>\n",
       "      <th>T-Loss</th>\n",
       "      <th>TNC</th>\n",
       "      <th>TS-TCC</th>\n",
       "      <th>TST</th>\n",
       "      <th>DTW</th>\n",
       "      <th>CNN</th>\n",
       "      <th>Encoder</th>\n",
       "      <th>FCN</th>\n",
       "      <th>MCDNN</th>\n",
       "      <th>MLP</th>\n",
       "      <th>ResNet</th>\n",
       "      <th>t-LeNet</th>\n",
       "      <th>TWIESN</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Adiac</th>\n",
       "      <td>0.762</td>\n",
       "      <td>0.675</td>\n",
       "      <td>0.726</td>\n",
       "      <td>0.767</td>\n",
       "      <td>0.550</td>\n",
       "      <td>0.604</td>\n",
       "      <td>0.393350</td>\n",
       "      <td>0.318159</td>\n",
       "      <td>0.841432</td>\n",
       "      <td>0.620460</td>\n",
       "      <td>0.391304</td>\n",
       "      <td>0.833248</td>\n",
       "      <td>0.022506</td>\n",
       "      <td>0.427621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArrowHead</th>\n",
       "      <td>0.857</td>\n",
       "      <td>0.766</td>\n",
       "      <td>0.703</td>\n",
       "      <td>0.737</td>\n",
       "      <td>0.771</td>\n",
       "      <td>0.703</td>\n",
       "      <td>0.716571</td>\n",
       "      <td>0.629714</td>\n",
       "      <td>0.843429</td>\n",
       "      <td>0.677714</td>\n",
       "      <td>0.784000</td>\n",
       "      <td>0.837714</td>\n",
       "      <td>0.302857</td>\n",
       "      <td>0.689143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Beef</th>\n",
       "      <td>0.767</td>\n",
       "      <td>0.667</td>\n",
       "      <td>0.733</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.500</td>\n",
       "      <td>0.633</td>\n",
       "      <td>0.766667</td>\n",
       "      <td>0.706667</td>\n",
       "      <td>0.680000</td>\n",
       "      <td>0.506667</td>\n",
       "      <td>0.713333</td>\n",
       "      <td>0.753333</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.526667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BeetleFly</th>\n",
       "      <td>0.900</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.8</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.700</td>\n",
       "      <td>0.900000</td>\n",
       "      <td>0.620000</td>\n",
       "      <td>0.910000</td>\n",
       "      <td>0.630000</td>\n",
       "      <td>0.880000</td>\n",
       "      <td>0.850000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.790000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BirdChicken</th>\n",
       "      <td>0.800</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.650</td>\n",
       "      <td>0.750</td>\n",
       "      <td>0.710000</td>\n",
       "      <td>0.510000</td>\n",
       "      <td>0.940000</td>\n",
       "      <td>0.540000</td>\n",
       "      <td>0.740000</td>\n",
       "      <td>0.880000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.620000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             TS2Vec T-Loss    TNC TS-TCC    TST    DTW       CNN   Encoder  \\\n",
       "Dataset                                                                      \n",
       "Adiac         0.762  0.675  0.726  0.767  0.550  0.604  0.393350  0.318159   \n",
       "ArrowHead     0.857  0.766  0.703  0.737  0.771  0.703  0.716571  0.629714   \n",
       "Beef          0.767  0.667  0.733    0.6  0.500  0.633  0.766667  0.706667   \n",
       "BeetleFly     0.900    0.8   0.85    0.8  1.000  0.700  0.900000  0.620000   \n",
       "BirdChicken   0.800   0.85   0.75   0.65  0.650  0.750  0.710000  0.510000   \n",
       "\n",
       "                  FCN     MCDNN       MLP    ResNet   t-LeNet    TWIESN  \n",
       "Dataset                                                                  \n",
       "Adiac        0.841432  0.620460  0.391304  0.833248  0.022506  0.427621  \n",
       "ArrowHead    0.843429  0.677714  0.784000  0.837714  0.302857  0.689143  \n",
       "Beef         0.680000  0.506667  0.713333  0.753333  0.200000  0.526667  \n",
       "BeetleFly    0.910000  0.630000  0.880000  0.850000  0.500000  0.790000  \n",
       "BirdChicken  0.940000  0.540000  0.740000  0.880000  0.500000  0.620000  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Join the two dataframes\n",
    "ucr_results = ucr_results_unsupervised.merge(ucr_results_supervised, on='Dataset')\n",
    "ucr_results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results path: /home/extra_scratch/mgoswami/moment_results/unsupervised_representation_learning\n"
     ]
    }
   ],
   "source": [
    "experiment_name = \"unsupervised_representation_learning\" \n",
    "\n",
    "results_path = os.path.join(\"/home/extra_scratch/mgoswami/moment_results/\", experiment_name)\n",
    "print(f\"Results path: {results_path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 29/29 [04:21<00:00,  9.03s/it]\n"
     ]
    }
   ],
   "source": [
    "dataset_with_results = [i for i in os.listdir(results_path) if 'results' in i]\n",
    "\n",
    "train_accuracy = {}\n",
    "test_accuracy = {}\n",
    "\n",
    "for dataset in tqdm(dataset_with_results, total=len(dataset_with_results)):\n",
    "    dataset_name = dataset.split(\"_\")[1][:-4]\n",
    "    full_path = os.path.join(results_path, dataset)\n",
    "    with open(full_path, \"rb\") as f:\n",
    "        r = pkl.load(f)\n",
    "    \n",
    "    train_accuracy[dataset_name] = r.train_accuracy\n",
    "    test_accuracy[dataset_name] = r.test_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MOMENT</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Dataset</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ArticularyWordRecognition</th>\n",
       "      <td>0.990000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AtrialFibrillation</th>\n",
       "      <td>0.200000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BasicMotions</th>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cricket</th>\n",
       "      <td>0.986111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DuckDuckGeese</th>\n",
       "      <td>0.600000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             MOMENT\n",
       "Dataset                            \n",
       "ArticularyWordRecognition  0.990000\n",
       "AtrialFibrillation         0.200000\n",
       "BasicMotions               1.000000\n",
       "Cricket                    0.986111\n",
       "DuckDuckGeese              0.600000"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "MOMENT_results = pd.DataFrame([test_accuracy]).T\n",
    "MOMENT_results.columns = ['MOMENT']\n",
    "MOMENT_results.index.name = 'Dataset'\n",
    "MOMENT_results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = MOMENT_results.merge(uea_results_unsupervised, on='Dataset')\n",
    "# results = results.merge(uea_results_supervised, on='Dataset')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MOMENT</th>\n",
       "      <th>TS2Vec</th>\n",
       "      <th>T-Loss</th>\n",
       "      <th>TNC</th>\n",
       "      <th>TS-TCC</th>\n",
       "      <th>TST</th>\n",
       "      <th>DTW</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ArticularyWordRecognition</th>\n",
       "      <td>0.990000</td>\n",
       "      <td>0.987</td>\n",
       "      <td>0.943</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.953</td>\n",
       "      <td>0.977</td>\n",
       "      <td>0.987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AtrialFibrillation</th>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.200</td>\n",
       "      <td>0.133</td>\n",
       "      <td>0.133</td>\n",
       "      <td>0.267</td>\n",
       "      <td>0.067</td>\n",
       "      <td>0.200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BasicMotions</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.975</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.975</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.975</td>\n",
       "      <td>0.975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cricket</th>\n",
       "      <td>0.986111</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.958</td>\n",
       "      <td>0.917</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DuckDuckGeese</th>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.680</td>\n",
       "      <td>0.650</td>\n",
       "      <td>0.460</td>\n",
       "      <td>0.380</td>\n",
       "      <td>0.620</td>\n",
       "      <td>0.600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EigenWorms</th>\n",
       "      <td>0.809160</td>\n",
       "      <td>0.847</td>\n",
       "      <td>0.840</td>\n",
       "      <td>0.840</td>\n",
       "      <td>0.779</td>\n",
       "      <td>0.748</td>\n",
       "      <td>0.618</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Epilepsy</th>\n",
       "      <td>0.992754</td>\n",
       "      <td>0.964</td>\n",
       "      <td>0.971</td>\n",
       "      <td>0.957</td>\n",
       "      <td>0.957</td>\n",
       "      <td>0.949</td>\n",
       "      <td>0.964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ERing</th>\n",
       "      <td>0.959259</td>\n",
       "      <td>0.874</td>\n",
       "      <td>0.133</td>\n",
       "      <td>0.852</td>\n",
       "      <td>0.904</td>\n",
       "      <td>0.874</td>\n",
       "      <td>0.133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EthanolConcentration</th>\n",
       "      <td>0.357414</td>\n",
       "      <td>0.308</td>\n",
       "      <td>0.205</td>\n",
       "      <td>0.297</td>\n",
       "      <td>0.285</td>\n",
       "      <td>0.262</td>\n",
       "      <td>0.323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FaceDetection</th>\n",
       "      <td>0.632804</td>\n",
       "      <td>0.501</td>\n",
       "      <td>0.513</td>\n",
       "      <td>0.536</td>\n",
       "      <td>0.544</td>\n",
       "      <td>0.534</td>\n",
       "      <td>0.529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FingerMovements</th>\n",
       "      <td>0.490000</td>\n",
       "      <td>0.480</td>\n",
       "      <td>0.580</td>\n",
       "      <td>0.470</td>\n",
       "      <td>0.460</td>\n",
       "      <td>0.560</td>\n",
       "      <td>0.530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HandMovementDirection</th>\n",
       "      <td>0.324324</td>\n",
       "      <td>0.338</td>\n",
       "      <td>0.351</td>\n",
       "      <td>0.324</td>\n",
       "      <td>0.243</td>\n",
       "      <td>0.243</td>\n",
       "      <td>0.231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Handwriting</th>\n",
       "      <td>0.308235</td>\n",
       "      <td>0.515</td>\n",
       "      <td>0.451</td>\n",
       "      <td>0.249</td>\n",
       "      <td>0.498</td>\n",
       "      <td>0.225</td>\n",
       "      <td>0.286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Heartbeat</th>\n",
       "      <td>0.721951</td>\n",
       "      <td>0.683</td>\n",
       "      <td>0.741</td>\n",
       "      <td>0.746</td>\n",
       "      <td>0.751</td>\n",
       "      <td>0.746</td>\n",
       "      <td>0.717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JapaneseVowels</th>\n",
       "      <td>0.716216</td>\n",
       "      <td>0.984</td>\n",
       "      <td>0.989</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.930</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.949</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Libras</th>\n",
       "      <td>0.850000</td>\n",
       "      <td>0.867</td>\n",
       "      <td>0.883</td>\n",
       "      <td>0.817</td>\n",
       "      <td>0.822</td>\n",
       "      <td>0.656</td>\n",
       "      <td>0.870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LSST</th>\n",
       "      <td>0.411192</td>\n",
       "      <td>0.537</td>\n",
       "      <td>0.509</td>\n",
       "      <td>0.595</td>\n",
       "      <td>0.474</td>\n",
       "      <td>0.408</td>\n",
       "      <td>0.551</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MotorImagery</th>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.510</td>\n",
       "      <td>0.580</td>\n",
       "      <td>0.500</td>\n",
       "      <td>0.610</td>\n",
       "      <td>0.500</td>\n",
       "      <td>0.500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NATOPS</th>\n",
       "      <td>0.827778</td>\n",
       "      <td>0.928</td>\n",
       "      <td>0.917</td>\n",
       "      <td>0.911</td>\n",
       "      <td>0.822</td>\n",
       "      <td>0.850</td>\n",
       "      <td>0.883</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PEMS-SF</th>\n",
       "      <td>0.895954</td>\n",
       "      <td>0.682</td>\n",
       "      <td>0.676</td>\n",
       "      <td>0.699</td>\n",
       "      <td>0.734</td>\n",
       "      <td>0.740</td>\n",
       "      <td>0.711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PenDigits</th>\n",
       "      <td>0.972270</td>\n",
       "      <td>0.989</td>\n",
       "      <td>0.981</td>\n",
       "      <td>0.979</td>\n",
       "      <td>0.974</td>\n",
       "      <td>0.560</td>\n",
       "      <td>0.977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PhonemeSpectra</th>\n",
       "      <td>0.232926</td>\n",
       "      <td>0.233</td>\n",
       "      <td>0.222</td>\n",
       "      <td>0.207</td>\n",
       "      <td>0.252</td>\n",
       "      <td>0.085</td>\n",
       "      <td>0.151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RacketSports</th>\n",
       "      <td>0.796053</td>\n",
       "      <td>0.855</td>\n",
       "      <td>0.855</td>\n",
       "      <td>0.776</td>\n",
       "      <td>0.816</td>\n",
       "      <td>0.809</td>\n",
       "      <td>0.803</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SelfRegulationSCP1</th>\n",
       "      <td>0.839590</td>\n",
       "      <td>0.812</td>\n",
       "      <td>0.843</td>\n",
       "      <td>0.799</td>\n",
       "      <td>0.823</td>\n",
       "      <td>0.754</td>\n",
       "      <td>0.775</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SelfRegulationSCP2</th>\n",
       "      <td>0.477778</td>\n",
       "      <td>0.578</td>\n",
       "      <td>0.539</td>\n",
       "      <td>0.550</td>\n",
       "      <td>0.533</td>\n",
       "      <td>0.550</td>\n",
       "      <td>0.539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SpokenArabicDigits</th>\n",
       "      <td>0.981355</td>\n",
       "      <td>0.988</td>\n",
       "      <td>0.905</td>\n",
       "      <td>0.934</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.923</td>\n",
       "      <td>0.963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>StandWalkJump</th>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.467</td>\n",
       "      <td>0.333</td>\n",
       "      <td>0.400</td>\n",
       "      <td>0.333</td>\n",
       "      <td>0.267</td>\n",
       "      <td>0.200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UWaveGestureLibrary</th>\n",
       "      <td>0.909375</td>\n",
       "      <td>0.906</td>\n",
       "      <td>0.875</td>\n",
       "      <td>0.759</td>\n",
       "      <td>0.753</td>\n",
       "      <td>0.575</td>\n",
       "      <td>0.903</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>InsectWingbeat</th>\n",
       "      <td>0.246160</td>\n",
       "      <td>0.466</td>\n",
       "      <td>0.156</td>\n",
       "      <td>0.469</td>\n",
       "      <td>0.264</td>\n",
       "      <td>0.105</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             MOMENT  TS2Vec  T-Loss    TNC  TS-TCC    TST  \\\n",
       "Dataset                                                                     \n",
       "ArticularyWordRecognition  0.990000   0.987   0.943  0.973   0.953  0.977   \n",
       "AtrialFibrillation         0.200000   0.200   0.133  0.133   0.267  0.067   \n",
       "BasicMotions               1.000000   0.975   1.000  0.975   1.000  0.975   \n",
       "Cricket                    0.986111   0.972   0.972  0.958   0.917  1.000   \n",
       "DuckDuckGeese              0.600000   0.680   0.650  0.460   0.380  0.620   \n",
       "EigenWorms                 0.809160   0.847   0.840  0.840   0.779  0.748   \n",
       "Epilepsy                   0.992754   0.964   0.971  0.957   0.957  0.949   \n",
       "ERing                      0.959259   0.874   0.133  0.852   0.904  0.874   \n",
       "EthanolConcentration       0.357414   0.308   0.205  0.297   0.285  0.262   \n",
       "FaceDetection              0.632804   0.501   0.513  0.536   0.544  0.534   \n",
       "FingerMovements            0.490000   0.480   0.580  0.470   0.460  0.560   \n",
       "HandMovementDirection      0.324324   0.338   0.351  0.324   0.243  0.243   \n",
       "Handwriting                0.308235   0.515   0.451  0.249   0.498  0.225   \n",
       "Heartbeat                  0.721951   0.683   0.741  0.746   0.751  0.746   \n",
       "JapaneseVowels             0.716216   0.984   0.989  0.978   0.930  0.978   \n",
       "Libras                     0.850000   0.867   0.883  0.817   0.822  0.656   \n",
       "LSST                       0.411192   0.537   0.509  0.595   0.474  0.408   \n",
       "MotorImagery               0.500000   0.510   0.580  0.500   0.610  0.500   \n",
       "NATOPS                     0.827778   0.928   0.917  0.911   0.822  0.850   \n",
       "PEMS-SF                    0.895954   0.682   0.676  0.699   0.734  0.740   \n",
       "PenDigits                  0.972270   0.989   0.981  0.979   0.974  0.560   \n",
       "PhonemeSpectra             0.232926   0.233   0.222  0.207   0.252  0.085   \n",
       "RacketSports               0.796053   0.855   0.855  0.776   0.816  0.809   \n",
       "SelfRegulationSCP1         0.839590   0.812   0.843  0.799   0.823  0.754   \n",
       "SelfRegulationSCP2         0.477778   0.578   0.539  0.550   0.533  0.550   \n",
       "SpokenArabicDigits         0.981355   0.988   0.905  0.934   0.970  0.923   \n",
       "StandWalkJump              0.400000   0.467   0.333  0.400   0.333  0.267   \n",
       "UWaveGestureLibrary        0.909375   0.906   0.875  0.759   0.753  0.575   \n",
       "InsectWingbeat             0.246160   0.466   0.156  0.469   0.264  0.105   \n",
       "\n",
       "                             DTW  \n",
       "Dataset                           \n",
       "ArticularyWordRecognition  0.987  \n",
       "AtrialFibrillation         0.200  \n",
       "BasicMotions               0.975  \n",
       "Cricket                    1.000  \n",
       "DuckDuckGeese              0.600  \n",
       "EigenWorms                 0.618  \n",
       "Epilepsy                   0.964  \n",
       "ERing                      0.133  \n",
       "EthanolConcentration       0.323  \n",
       "FaceDetection              0.529  \n",
       "FingerMovements            0.530  \n",
       "HandMovementDirection      0.231  \n",
       "Handwriting                0.286  \n",
       "Heartbeat                  0.717  \n",
       "JapaneseVowels             0.949  \n",
       "Libras                     0.870  \n",
       "LSST                       0.551  \n",
       "MotorImagery               0.500  \n",
       "NATOPS                     0.883  \n",
       "PEMS-SF                    0.711  \n",
       "PenDigits                  0.977  \n",
       "PhonemeSpectra             0.151  \n",
       "RacketSports               0.803  \n",
       "SelfRegulationSCP1         0.775  \n",
       "SelfRegulationSCP2         0.539  \n",
       "SpokenArabicDigits         0.963  \n",
       "StandWalkJump              0.200  \n",
       "UWaveGestureLibrary        0.903  \n",
       "InsectWingbeat               NaN  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lrrrrrrr}\n",
      "\\toprule\n",
      " & MOMENT & TS2Vec & T-Loss & TNC & TS-TCC & TST & DTW \\\\\n",
      "\\midrule\n",
      "count & 29.000 & 29.000 & 29.000 & 29.000 & 29.000 & 29.000 & 28.000 \\\\\n",
      "mean & 0.670 & 0.694 & 0.646 & 0.660 & 0.657 & 0.605 & 0.638 \\\\\n",
      "std & 0.274 & 0.255 & 0.296 & 0.267 & 0.263 & 0.294 & 0.296 \\\\\n",
      "min & 0.200 & 0.200 & 0.133 & 0.133 & 0.243 & 0.067 & 0.133 \\\\\n",
      "25% & 0.411 & 0.501 & 0.451 & 0.469 & 0.460 & 0.408 & 0.456 \\\\\n",
      "50% & 0.722 & 0.683 & 0.676 & 0.746 & 0.751 & 0.620 & 0.664 \\\\\n",
      "75% & 0.909 & 0.928 & 0.905 & 0.911 & 0.904 & 0.850 & 0.914 \\\\\n",
      "max & 1.000 & 0.989 & 1.000 & 0.979 & 1.000 & 1.000 & 1.000 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(results.describe().to_latex(float_format=\"%.3f\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "results.to_latex(\"../../assets/results/zero_shot/multi_variate_classification.tex\", multicolumn_format='c', float_format=\"%.3f\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read TimesNet and GPT4TS results\n",
    "timesnet_gpt4ts_results = pd.read_csv('../../assets/results/finetuning/timesnet_gpt4ts_classification.csv')\n",
    "timesnet_gpt4ts_results = timesnet_gpt4ts_results.drop(columns=['Wandb Run (TimesNet)', 'Wandb Run (GPT4TS)'])\n",
    "timesnet_results = timesnet_gpt4ts_results[['Dataset', 'TimesNet Test Accuracy']].set_index('Dataset')\n",
    "timesnet_results.columns = ['TimesNet']\n",
    "gpt4ts_results = timesnet_gpt4ts_results[['Dataset', 'GPT4TS Test Accuracy']].set_index('Dataset')\n",
    "gpt4ts_results.columns = ['GPT4TS']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = MOMENT_results.merge(ucr_results, on='Dataset')\n",
    "results = results.merge(timesnet_results, on='Dataset')\n",
    "results = results.merge(gpt4ts_results, on='Dataset')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = MOMENT_results.merge(ucr_results, on='Dataset')\n",
    "results = results.merge(timesnet_results, on='Dataset')\n",
    "results = results.merge(gpt4ts_results, on='Dataset')\n",
    "results = results[[\n",
    "    'MOMENT', 'TimesNet', 'GPT4TS', \n",
    "    'TS2Vec', 'T-Loss', 'TNC', 'TS-TCC', 'TST', \n",
    "    'CNN', 'Encoder', 'FCN', 'MCDNN', 'MLP', 'ResNet', 't-LeNet', 'TWIESN',\n",
    "    'DTW']]\n",
    "results.to_csv(\"../../assets/results/zero_shot/unsupervised_representation_learning.csv\", index=False)\n",
    "results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MOMENT</th>\n",
       "      <th>TS2Vec</th>\n",
       "      <th>T-Loss</th>\n",
       "      <th>TNC</th>\n",
       "      <th>TS-TCC</th>\n",
       "      <th>TST</th>\n",
       "      <th>DTW</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ArticularyWordRecognition</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>7.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AtrialFibrillation</th>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.5</td>\n",
       "      <td>5.5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BasicMotions</th>\n",
       "      <td>2.0</td>\n",
       "      <td>5.5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.5</td>\n",
       "      <td>5.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cricket</th>\n",
       "      <td>3.0</td>\n",
       "      <td>4.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.5</td>\n",
       "      <td>1.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DuckDuckGeese</th>\n",
       "      <td>4.5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           MOMENT  TS2Vec  T-Loss  TNC  TS-TCC  TST  DTW\n",
       "Dataset                                                                 \n",
       "ArticularyWordRecognition     1.0     2.5     7.0  5.0     6.0  4.0  2.5\n",
       "AtrialFibrillation            3.0     3.0     5.5  5.5     1.0  7.0  3.0\n",
       "BasicMotions                  2.0     5.5     2.0  5.5     2.0  5.5  5.5\n",
       "Cricket                       3.0     4.5     4.5  6.0     7.0  1.5  1.5\n",
       "DuckDuckGeese                 4.5     1.0     2.0  6.0     7.0  3.0  4.5"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Average rank of each method on each dataset\n",
    "average_rank = results.rank(axis=1, method='average', ascending=False)\n",
    "average_rank.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MOMENT</th>\n",
       "      <th>TS2Vec</th>\n",
       "      <th>T-Loss</th>\n",
       "      <th>TNC</th>\n",
       "      <th>TS-TCC</th>\n",
       "      <th>TST</th>\n",
       "      <th>DTW</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>29.000000</td>\n",
       "      <td>29.000000</td>\n",
       "      <td>29.000000</td>\n",
       "      <td>29.000000</td>\n",
       "      <td>29.00000</td>\n",
       "      <td>29.000000</td>\n",
       "      <td>28.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>3.465517</td>\n",
       "      <td>2.862069</td>\n",
       "      <td>3.603448</td>\n",
       "      <td>4.362069</td>\n",
       "      <td>4.12069</td>\n",
       "      <td>5.068966</td>\n",
       "      <td>4.428571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.986252</td>\n",
       "      <td>1.831747</td>\n",
       "      <td>2.106026</td>\n",
       "      <td>1.597412</td>\n",
       "      <td>2.05572</td>\n",
       "      <td>1.850310</td>\n",
       "      <td>1.642685</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>1.500000</td>\n",
       "      <td>1.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.00000</td>\n",
       "      <td>3.500000</td>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.500000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.00000</td>\n",
       "      <td>5.500000</td>\n",
       "      <td>4.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>5.000000</td>\n",
       "      <td>3.500000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.500000</td>\n",
       "      <td>6.00000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>5.625000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.00000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          MOMENT     TS2Vec     T-Loss        TNC    TS-TCC        TST  \\\n",
       "count  29.000000  29.000000  29.000000  29.000000  29.00000  29.000000   \n",
       "mean    3.465517   2.862069   3.603448   4.362069   4.12069   5.068966   \n",
       "std     1.986252   1.831747   2.106026   1.597412   2.05572   1.850310   \n",
       "min     1.000000   1.000000   1.000000   1.000000   1.00000   1.500000   \n",
       "25%     2.000000   1.000000   2.000000   3.000000   2.00000   3.500000   \n",
       "50%     3.000000   2.500000   4.000000   5.000000   5.00000   5.500000   \n",
       "75%     5.000000   3.500000   5.000000   5.500000   6.00000   7.000000   \n",
       "max     7.000000   7.000000   7.000000   7.000000   7.00000   7.000000   \n",
       "\n",
       "             DTW  \n",
       "count  28.000000  \n",
       "mean    4.428571  \n",
       "std     1.642685  \n",
       "min     1.500000  \n",
       "25%     3.000000  \n",
       "50%     4.500000  \n",
       "75%     5.625000  \n",
       "max     7.000000  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "average_rank.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute number so wins / ties / losses for each method \n",
    "wins = (results.rank(axis=1, method='average', ascending=True) - 1).sum(axis=0)\n",
    "losses = (results.rank(axis=1, method='average', ascending=False) - 1).sum(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MOMENT    101.5\n",
       "TS2Vec    119.0\n",
       "T-Loss     97.5\n",
       "TNC        75.5\n",
       "TS-TCC     82.5\n",
       "TST        55.0\n",
       "DTW        72.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wins"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MOMENT     71.5\n",
       "TS2Vec     54.0\n",
       "T-Loss     75.5\n",
       "TNC        97.5\n",
       "TS-TCC     90.5\n",
       "TST       118.0\n",
       "DTW        96.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "losses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results.to_latex(\"../../assets/results/zero_shot/classification.tex\", multicolumn_format='c', float_format=\"%.3f\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "boxprops = dict(linestyle='-', linewidth=1, color='k')\n",
    "flierprops = dict(marker='o', markersize=12, markeredgecolor='darkgreen')\n",
    "medianprops = dict(linestyle='-', linewidth=2, color='blue')\n",
    "meanpointprops = dict(marker='D', markeredgecolor='black',\n",
    "                      markerfacecolor='firebrick')\n",
    "meanlineprops = dict(linestyle='--', linewidth=2, color='red')\n",
    "\n",
    "model_names = results.columns.tolist()\n",
    "\n",
    "fig = plt.figure(figsize=(10, 6))  # Specify the size of the figure\n",
    "_ = plt.boxplot(results,\n",
    "                labels=model_names, \n",
    "                meanline=True, \n",
    "                showmeans=True, \n",
    "                notch=True,\n",
    "                bootstrap=10000,\n",
    "                flierprops=flierprops,\n",
    "                meanprops=meanlineprops, \n",
    "                boxprops=boxprops,\n",
    "                medianprops=medianprops,\n",
    "                )\n",
    "\n",
    "plt.grid(color='lightgray', linestyle='--', linewidth=0.5) \n",
    "plt.ylabel(\"Accuracy\", fontsize=14)\n",
    "plt.xticks(rotation=45, ha='right', fontsize=14)\n",
    "plt.yticks(fontsize=14)\n",
    "plt.title(\"Accuracy on UCR datasets\", fontsize=16)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results.reset_index(inplace=True)\n",
    "long_results = results.melt(id_vars=['Dataset'], value_vars=model_names)\n",
    "long_results.columns= ['dataset_name', 'classifier_name', 'accuracy']\n",
    "long_results = long_results[['classifier_name', 'dataset_name', 'accuracy']]\n",
    "long_results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt, p_values, average_ranks = draw_cd_diagram(df_perf = long_results, alpha = 0.05, labels='Accuracy')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Results summary\n",
    "columns = ['MOMENT', 'TS2Vec', 'T-Loss', 'TNC', 'TS-TCC', 'TST', \n",
    " 'CNN', 'Encoder', 'FCN', 'MCDNN', 'MLP', 'ResNet', 't-LeNet', 'TWIESN', \n",
    " 'DTW']\n",
    "results[columns].fillna(0).describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "summary = pd.concat(\n",
    "    [results[columns].mean(axis=0, skipna=True).astype(np.float16),\n",
    "     results[columns].median(axis=0, skipna=True).astype(np.float16),\n",
    "     results[columns].std(axis=0, skipna=True).astype(np.float16)], axis=1).T\n",
    "summary.index = ['Mean', 'Median', 'Std.']\n",
    "summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(summary.to_latex(float_format=\"%.3f\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Datasets with worst performance in comarison to TS2Vec\n",
    "(results['TS2Vec'] - results['MOMENT']).sort_values(ascending=False)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "summary = pd.read_csv(\"../../assets/data/summaryUnivariate.csv\")\n",
    "summary.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# low_accuracy_datasets = sorted(test_accuracy, key=test_accuracy.get, reverse=False)[:15]\n",
    "low_accuracy_datasets = (results['TS2Vec'] - results['MOMENT']).sort_values(ascending=False)[:10].index.tolist()\n",
    "low_accuracy_datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analyze low accuracy datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "summary[summary[\"problem\"].isin(low_accuracy_datasets)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Transform this from a dictionary to a dataframe\n",
    "accuracies = pd.DataFrame(data=[test_accuracy, train_accuracy]).T\n",
    "accuracies.columns = [\"Test accuracy\", \"Train accuracy\"]\n",
    "accuracies = accuracies.merge(summary, left_index=True, right_on=\"problem\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.scatter(accuracies.loc[:, \"Test accuracy\"], accuracies.loc[:, \"numTrainCases\"])\n",
    "plt.xlabel(\"Test accuracy\", fontsize=16)\n",
    "plt.ylabel(\"Number of training cases\", fontsize=16)\n",
    "plt.title(\"Test accuracy vs number of training cases\", fontsize=18)\n",
    "plt.ylim(0, 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.scatter(accuracies.loc[:, \"Test accuracy\"], accuracies.loc[:, \"seriesLength\"])\n",
    "plt.xlabel(\"Test accuracy\", fontsize=16)\n",
    "plt.ylabel(\"Series Length\", fontsize=16)\n",
    "plt.ylim(0, 600)\n",
    "plt.title(\"Test accuracy vs series length\", fontsize=18)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fine-tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "from moment.utils.config import Config\n",
    "from moment.utils.utils import parse_config\n",
    "from moment.data.dataloader import get_timeseries_dataloader\n",
    "from moment.models.base import BaseModel\n",
    "from moment.models.moment import MOMENT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_dataloaders(args):\n",
    "    args.dataset_names = args.full_file_path_and_name\n",
    "    args.data_split = 'train'\n",
    "    train_dataloader = get_timeseries_dataloader(args=args)\n",
    "    args.data_split = 'test'\n",
    "    test_dataloader = get_timeseries_dataloader(args=args)\n",
    "    args.data_split = 'val'\n",
    "    val_dataloader = get_timeseries_dataloader(args=args)\n",
    "    return train_dataloader, test_dataloader, val_dataloader\n",
    "\n",
    "def load_pretrained_moment(args,\n",
    "                         pretraining_task_name: str = \"pre-training\"):\n",
    "    args.task_name = pretraining_task_name\n",
    "        \n",
    "    checkpoint = BaseModel.load_pretrained_weights(\n",
    "        run_name=args.pretraining_run_name, \n",
    "        opt_steps=args.pretraining_opt_steps)\n",
    "    \n",
    "    pretrained_model = MOMENT(configs=args)\n",
    "    pretrained_model.load_state_dict(checkpoint[\"model_state_dict\"])\n",
    "    \n",
    "    return pretrained_model\n",
    "\n",
    "def freeze_model_parameters(args, model):\n",
    "    if args.finetuning_mode == 'linear-probing':\n",
    "        for name, param in model.named_parameters():\n",
    "            name = name.lower()\n",
    "            if 'ln' in name or 'norm' in name or 'layer_norm' in name:\n",
    "                param.requires_grad = True\n",
    "            elif 'wpe' in name or 'position_embeddings' in name or 'pos_drop' in name:\n",
    "                param.requires_grad = True\n",
    "            elif 'mlp' in name or 'densereludense' in name:\n",
    "                param.requires_grad = False\n",
    "            elif 'attn' in name or 'selfattention' in name:\n",
    "                param.requires_grad = False\n",
    "            elif 'head' in name:\n",
    "                param.requires_grad = True\n",
    "            elif 'patch_embedding' in name:\n",
    "                param.requires_grad = True\n",
    "            else:\n",
    "                param.requires_grad = False\n",
    "\n",
    "    print(\"====== Frozen parameter status ======\")\n",
    "    for name, param in model.named_parameters():\n",
    "        if param.requires_grad:\n",
    "            print(\"Not frozen:\", name)\n",
    "        else:\n",
    "            print(\"Frozen:\", name)\n",
    "    print(\"=====================================\")\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config_path = \"../../configs/classification/unsupervised_representation_learning.yaml\"\n",
    "DEFAULT_CONFIG_PATH = \"../../configs/default.yaml\"\n",
    "gpu_id = 0\n",
    "\n",
    "# Load arguments and parse them\n",
    "config = Config(config_file_path=config_path, \n",
    "            default_config_file_path=DEFAULT_CONFIG_PATH).parse()\n",
    "\n",
    "config['device'] = torch.device('cuda:{}'.format(gpu_id)) if torch.cuda.is_available() else 'cpu'\n",
    "args = parse_config(config)\n",
    "\n",
    "args.full_file_path_and_name = '/TimeseriesDatasets/classification/UCR/Beef/Beef_TEST.ts'\n",
    "args.max_epoch = 20\n",
    "args.batch_size = 16\n",
    "args.init_lr = 0.0001\n",
    "args.upsampling_type = 'interpolate' \n",
    "# args.upsampling_type = 'pad' # 'interpolate' 'pad\n",
    "args.finetuning_mode = 'linear-probing'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = load_pretrained_moment(args)\n",
    "model = freeze_model_parameters(args, model)\n",
    "model.to(args.device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "args.task_name = \"classification\"\n",
    "train_dataloader, test_dataloader, val_dataloader = get_dataloaders(args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trues = []\n",
    "preds = []\n",
    "input_masks = []\n",
    "\n",
    "with torch.no_grad():\n",
    "    for batch_x in tqdm(test_dataloader, total=len(test_dataloader)):\n",
    "        timeseries = batch_x.timeseries.float().to(args.device)\n",
    "        input_mask = batch_x.input_mask.long().to(args.device)\n",
    "\n",
    "        outputs = model.reconstruct(\n",
    "            x_enc=timeseries, input_mask=input_mask)\n",
    "        \n",
    "        preds.append(outputs.reconstruction.detach().cpu().numpy())\n",
    "        trues.append(timeseries.detach().cpu().numpy())\n",
    "        input_masks.append(input_mask.detach().cpu().numpy())\n",
    "\n",
    "    trues = np.concatenate(trues, axis=0).squeeze()\n",
    "    preds = np.concatenate(preds, axis=0).squeeze()\n",
    "    input_masks = np.concatenate(input_masks, axis=0).squeeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx = np.random.randint(0, len(trues))\n",
    "plt.title(f\"idx: {idx}\")\n",
    "plt.plot(trues[idx], label=\"True\")\n",
    "plt.plot(preds[idx], label=\"Predicted\")\n",
    "plt.plot(input_masks[idx], label=\"Input mask\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "from moment.utils.short_univariate_classification_datasets import \\\n",
    "    short_univariate_classification_datasets\n",
    "\n",
    "args.task_name = \"classification\"\n",
    "features = []\n",
    "for dataset_name in tqdm(short_univariate_classification_datasets):\n",
    "    args.full_file_path_and_name = dataset_name\n",
    "    train_dataloader, test_dataloader, val_dataloader = get_dataloaders(args)\n",
    "    \n",
    "    train_data = np.concatenate([\n",
    "        train_dataloader.dataset.data, val_dataloader.dataset.data], axis=1)\n",
    "    labels = np.concatenate([train_dataloader.dataset.labels, val_dataloader.dataset.labels])\n",
    "    num_classes = len(np.unique(labels.flatten()))\n",
    "    \n",
    "    len_timeseries, n_train = train_data.shape\n",
    "    len_timeseries, n_test = test_dataloader.dataset.data.shape\n",
    "\n",
    "    features.append([dataset_name.split(\"/\")[-2], n_train, n_test, len_timeseries, num_classes])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = pd.DataFrame(features, columns=['problem', 'num_train', 'num_test', 'series_length', 'num_classes'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_comparison = summary.merge(features, on='problem')\n",
    "feature_comparison.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_bool = feature_comparison.numClasses.astype(int) != feature_comparison.num_classes.astype(int)\n",
    "train_bool = feature_comparison.num_train.astype(int) != feature_comparison.numTrainCases.astype(int)\n",
    "test_bool = feature_comparison.num_test.astype(int) != feature_comparison.numTestCases.astype(int)\n",
    "length_bool = feature_comparison.series_length.astype(int) != feature_comparison.seriesLength.astype(int)\n",
    "\n",
    "print(' Num. classes:', class_bool.sum())\n",
    "print('  Train cases:', train_bool.sum())\n",
    "print('   Test cases:', test_bool.sum())\n",
    "print('Series length:', length_bool.sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mismatches = feature_comparison[train_bool | test_bool | length_bool]\n",
    "mismatches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results.merge(mismatches, right_on='problem', left_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "import wandb\n",
    "from torch import nn\n",
    "from torch import optim\n",
    "from tqdm import trange, tqdm\n",
    "\n",
    "from moment.common import PATHS\n",
    "\n",
    "\n",
    "def train(args, model, train_dataloader):\n",
    "        n_train_epochs = args.max_epoch\n",
    "        \n",
    "        # Training loop\n",
    "        tr_loss = 0\n",
    "        \n",
    "        optimizer = optim.AdamW(model.parameters(), \n",
    "                                lr=args.init_lr,\n",
    "                                weight_decay=args.weight_decay)\n",
    "\n",
    "        criterion = nn.MSELoss() \n",
    "\n",
    "        logger = wandb.init(\n",
    "            project=\"Time-series Foundation Model\",\n",
    "            dir=PATHS.WANDB_DIR)\n",
    "        \n",
    "        for epoch in trange(n_train_epochs):\n",
    "            for batch in tqdm(train_dataloader, total=len(train_dataloader)):\n",
    "                timeseries = batch.timeseries.float().to(args.device)\n",
    "                input_mask = batch.input_mask.long().to(args.device)\n",
    "\n",
    "                model.train()\n",
    "                # Training step\n",
    "                outputs = model.reconstruct(x_enc=timeseries, \n",
    "                                input_mask=input_mask, mask=None)\n",
    "                \n",
    "                loss = criterion(outputs.reconstruction, timeseries)\n",
    "\n",
    "                if not np.isnan(float(loss)):\n",
    "                    loss.backward()\n",
    "                \n",
    "                logger.log({\n",
    "                     'step_loss_train': loss.item(),\n",
    "                     'lr': optimizer.param_groups[0]['lr']})\n",
    "                \n",
    "                nn.utils.clip_grad_norm_(model.parameters(), args.max_norm)\n",
    "                    \n",
    "                optimizer.step()\n",
    "                optimizer.zero_grad()\n",
    "                \n",
    "                tr_loss += loss.detach().cpu().numpy()\n",
    "\n",
    "        logger.finish()\n",
    "\n",
    "        return model\n",
    "\n",
    "def get_embeddings_and_labels(model : torch.nn.Module, \n",
    "                              dataloader : torch.utils.data.DataLoader,\n",
    "                              device : torch.device, \n",
    "                              enable_batchwise_pbar : bool = False):\n",
    "    model = model.to(device)\n",
    "    model.eval()\n",
    "\n",
    "    embeddings = []\n",
    "    labels = []\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for batch_x in tqdm(dataloader, total=len(dataloader), \n",
    "                            disable=(not enable_batchwise_pbar)):\n",
    "            timeseries = batch_x.timeseries.float().to(device)\n",
    "            input_mask = batch_x.input_mask.long().to(device)\n",
    "\n",
    "            outputs = model.embed(x_enc=timeseries, input_mask=input_mask, reduction='mean')\n",
    "            \n",
    "            embeddings_ = outputs.embeddings.detach().cpu().numpy()\n",
    "            embeddings.append(embeddings_)\n",
    "            labels.append(batch_x.labels)\n",
    "\n",
    "        embeddings = np.concatenate(embeddings, axis=0)\n",
    "        labels = np.concatenate(labels, axis=0).squeeze()\n",
    " \n",
    "    return embeddings, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model = train(args, model, train_dataloader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from moment.models.statistical_classifiers import fit_svm\n",
    "\n",
    "train_embeddings, train_labels = get_embeddings_and_labels(\n",
    "        model=model, dataloader=train_dataloader, \n",
    "        device=torch.device(args.device), \n",
    "        enable_batchwise_pbar=False)\n",
    "    \n",
    "test_embeddings, test_labels = get_embeddings_and_labels(\n",
    "    model=model, dataloader=test_dataloader, \n",
    "    device=torch.device(args.device), \n",
    "    enable_batchwise_pbar=False)\n",
    "\n",
    "val_embeddings, val_labels = get_embeddings_and_labels(\n",
    "    model=model, dataloader=val_dataloader, \n",
    "    device=torch.device(args.device), \n",
    "    enable_batchwise_pbar=False)\n",
    "\n",
    "train_embeddings = np.concatenate([train_embeddings, val_embeddings], axis=0)\n",
    "train_labels = np.concatenate([train_labels, val_labels], axis=0)\n",
    "\n",
    "classifier = fit_svm(features=train_embeddings, y=train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluate the model\n",
    "test_accuracy = classifier.score(test_embeddings, test_labels)\n",
    "print(f\"Test accuracy: {test_accuracy}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results[results.index == 'Beef']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SMALL_IMAGE_DATASETS = ['Crop', 'MedicalImages', 'SwedishLeaf', \n",
    "                        'FacesUCR', 'FaceAll', 'Adiac', 'ArrowHead']\n",
    "SMALL_SPECTRO_DATASETS = ['Wine', 'Strawberry', 'Coffee', 'Ham', 'Meat', 'Beef']\n",
    "\n",
    "['ProximalPhalanxTW', 'ProximalPhalanxOutlineCorrect', 'ProximalPhalanxOutlineAgeGroup',\n",
    " 'PhalangesOutlinesCorrect', 'MiddlePhalanxTW', 'MiddlePhalanxOutlineCorrect', 'MiddlePhalanxOutlineAgeGroup',\n",
    " 'DistalPhalanxTW', 'DistalPhalanxOutlineCorrect', 'DistalPhalanxOutlineAgeGroup']"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
